import aiohttp
import asyncio
import requests
import re
from lxml import etree
import urllib
from aiohttp import ClientSession, TCPConnector
from 亚马逊目录.useragent import ua as jim


class Obtain():
    def __init__(self):
        self.headers = {
            'cache-control': 'no-cache',
            'cookie': 'session-id=130-2322939-7739406; session-id-time=2082787201l; i18n-prefs=USD; ubid-main=135-3888102-9406257; sp-cdn="L5Z9:TW"; session-token=L547CyHrI797TpcVGi9GZKHjNTXixkBViL+zw1hgOWD6X6DSP4yLBdMHJiGWKOaL3q1PvyfQ14RXG79syhJiuKVTlnwzw2rPqg1zy9G7Vwdp6j2CwZVNfIXjJeR60Y6fz+3odT3/90uVpQePMln+8T2UKmMNgfinKVx7G/gj2z2346/Rhgy71AdS4IrbLIA3; csm-hit=tb:5S3HN6WV2W7GKR3D6SP4+s-9EDEM9Q2YB1A1EBSPD2E|1629943795000&t:1629943795000&adb:adblk_no',
            'downlink': '8.75',
            'ect': '4g',
            'pragma': 'no-cache',
            'referer': 'https://www.amazon.com/Best-Sellers-Appstore-Android-Music-Audio/zgbs/mobile-apps/9408771011/ref=zg_bs_nav_mas_1_mas',
            'rtt': '250',
            'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
            'sec-ch-ua-mobile': '?0',
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'same-origin',
            'sec-fetch-user': '?1',
            'upgrade-insecure-requests': '1',
            'user-agent': '{}'.format(jim())}
        self.timeout = aiohttp.ClientTimeout(total=60)
        self.saves = open('./amazon.csv', 'w', encoding='utf-8')

    def level_one(self, url):

        # async with aiohttp.ClientSession() as session:
        #     async with session.get(url=url, headers=self.headers) as response:
        response = requests.get(url, headers=self.headers, verify=False).content.decode()
        # print(response)#第一层源代码
        # print('-'*10)
        l_HTML1 = etree.HTML(response)
        # print(l_HTML1)
        li_link = l_HTML1.xpath("//div[@id='zg_left_col2']/ul[@id='zg_browseRoot']/ul/li/a/@href")
        li_name = l_HTML1.xpath("//div[@id='zg_left_col2']/ul[@id='zg_browseRoot']/ul/li/a/text()")
        list1 = []
        for i in range(len(li_name)):
            print('-' * 20)
            print(li_link[i], li_name[i])
            print('*' * 20)
            link_name = (li_link[i], li_name[i])
            list1.append(link_name)
        # self.level_two(li_link[i])

        return list1

    async def level_two(self, url, name, semaphore):
        name = list(name)
        async with semaphore:
            async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=50, verify_ssl=False),
                                             timeout=self.timeout) as session:
                async with session.get(url=url, headers=self.headers) as response:
                    response = await response.text()
                    l_HTML2 = etree.HTML(response)
                    # print(l_HTML2)
                    print(url)
                    print('*' * 20)
                    li_link = l_HTML2.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/li/a/@href")
                    li_name = l_HTML2.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/li/a/text()")

                    if l_HTML2.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/li/span[@class='zg_selected']/text()") == []:
                        for i2 in range(len(li_link)):
                            print(li_link[i2], li_name[i2])
                            print('#2' * 20)
                            name.append(li_name[i2])
                            await self.level_three(li_link[i2], name, semaphore)
                            name = name[:-1]
                    else:
                        self.saves.write(name)


    async def level_three(self, url, name, semaphore):
        try:
            async with semaphore:
                async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=50, verify_ssl=False),
                                                 timeout=self.timeout) as session:
                    async with session.get(url=url, headers=self.headers) as response:
                        response = await response.text()
                        l_HTML3 = etree.HTML(response)
                        # print(l_HTML3)
                        # print(url)
                        # print('*' * 20)
                        li_link = l_HTML3.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/li/a/@href")
                        li_name = l_HTML3.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/li/a/text()")
                        if l_HTML3.xpath(
                                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/li/span[@class='zg_selected']/text()") == []:
                            for i3 in range(len(li_link)):
                                print(li_link[i3], li_name[i3])
                                print('#3' * 20)
                                name.append(li_name[i3])
                                await self.level_four(li_link[i3], name, semaphore)
                                name = name[:-1]
                        else:
                            self.saves.write(name)

        except Exception as g:
            await self.level_three(url, name, semaphore)

    async def level_four(self, url, name, semaphore):
        try:
            async with semaphore:
                async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=50, verify_ssl=False),
                                                 timeout=self.timeout) as session:
                    async with session.get(url=url, headers=self.headers) as response:
                        response = await response.text()
                        l_HTML4 = etree.HTML(response)
                        # print(l_HTML4)
                        # print(url)
                        # print('*' * 20)
                        li_link = l_HTML4.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/li/a/@href")
                        li_name = l_HTML4.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/li/a/text()")
                        if l_HTML4.xpath(
                                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/li/span[@class='zg_selected']/text()") == []:
                            for i4 in range(len(li_link)):
                                print(li_link[i4], li_name[i4])
                                print('#4' * 20)
                                name.append(li_name[i4])
                                await self.level_five(li_link[i4], name, semaphore)
                                name = name[:-1]
                        else:
                            self.saves.write(name)

        except  Exception as g:
            await self.level_four(url, name, semaphore)

    async def level_five(self, url, name, semaphore):
        try:
            async with semaphore:
                async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=50, verify_ssl=False),
                                                 timeout=self.timeout) as session:
                    async with session.get(url=url, headers=self.headers) as response:
                        response = await response.text()
                        l_HTML5 = etree.HTML(response)
                        # print(l_HTML3)
                        # print(url)
                        # print('*' * 20)
                        li_link = l_HTML5.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/li/a/@href")
                        li_name = l_HTML5.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/li/a/text()")
                        if l_HTML5.xpath(
                                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/li/span[@class='zg_selected']/text()") == []:
                            for i5 in range(len(li_link)):
                                print(li_link[i5], li_name[i5])
                                print('#5' * 20)
                                name.append(li_name[i5])
                                await self.level_six(li_link[i5], name, semaphore)
                                name = name[:-1]
                        else:
                            self.saves.write(name)
                            # name = name[:-1]
        except Exception as g:
            await self.level_five(url, name, semaphore)

    async def level_six(self, url, name, semaphore):
        try:
            async with semaphore:
                async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=50, verify_ssl=False),
                                                 timeout=self.timeout) as session:
                    async with session.get(url=url, headers=self.headers) as response:
                        response = await response.text()
                        l_HTML6 = etree.HTML(response)
                        # print(l_HTML3)
                        # print(url)
                        # print('*' * 20)
                        li_link = l_HTML6.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/ul/li/a/@href")
                        li_name = l_HTML6.xpath(
                            "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/ul/li/a/text()")
                        if l_HTML6.xpath(
                                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/ul/li/span[@class='zg_selected']/text()"):
                            for i6 in range(len(li_link)):
                                print(li_link[i6], li_name[i6])
                                print('#6' * 20)
                                name.append(li_name[i6])
                        else:
                            self.saves.write(name)

        except Exception as g:
            await self.level_six(url, name, semaphore)


if __name__ == '__main__':
    obtain = Obtain()
    semaphore = asyncio.Semaphore(100)

    url = 'https://www.amazon.com/Best-Sellers/zgbs/ref=zg_bs_unv_mas_0_9408771011_2'
    task = [asyncio.ensure_future(obtain.level_two(url1[0], url1[1], semaphore, )) for url1 in obtain.level_one(url)]
    loop = asyncio.get_event_loop()
    loop.run_until_complete(asyncio.wait(task))
